#!/usr/bin/perl

my %dictionary;
my %words;

sub count {
	$filehandle = shift;
	while (<$filehandle>) {
		$_ =~ s/\x0a//g;
		$_ =~ s/\x0d//g;
		$_ =~ s/\t/ /g; # deal with tabs
		$_ =~ s/[^[:print:]]+//g; # deal with non-printable characters
		$_ =~ s/[,.:;!`'"]/ /g; # deal with stupid punctuation
		if (($filename =~ /.*nitter.*/) || ($filename =~ /.*blogspot.*/)) { # deal with nitter/blogspot specific formatting
			$_ =~ s/\[[a-z]+_[a-z]+\]//g;
		}
		foreach $word (split(/ /, $_)) {
			if ($word ne "") {
				if ($word !~ /h..p[s]*:\/\//) { # deal with links
					$word =~ s/(.*)_[0-9a-f]{5}$/$1/g; # deal with DFIR reports that appear to result in a lot of <common string>_<5 digits of hex>
					if ((!$dictionary{$word}) && ($word =~ /[[:alnum:]]{2}[\/_][[:alnum:]]{2}/)) { # eliminate noise
						if ($words{$word}) {
							$words{$word} ++;
						} else {
							$words{$word} = 1;
						}
					}
				}
			}
		}
	}
}

open($filehandle, "</usr/share/dict/words");
while (<$filehandle>) {
	$_ =~ s/\x0a//g;
	$_ =~ s/\x0d//g;
	$dictionary{$_} = 1;
}
close($filehandle);
%words = ();
for $filename (glob("articles/*.html")) {
	open($filehandle, "w3m -dump " . $filename . "|");
	count($filehandle);
	close($filehandle);
}
for $filename (glob("articles/*.pdf")) {
	open($filehandle, "pdftotext " . $filename . " -|");
	count($filehandle);
	close($filehandle);
}
open($filehandle, ">all-words.txt");
for $word (keys %words) {
	print $filehandle $words{$word} . "\t" . $word . "\n"
}
close($filehandle);
